{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cifar-10-python.tar.gz']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../input\"))\n",
    "\n",
    "import time\n",
    "\n",
    "# import pytorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import SGD,Adam,lr_scheduler\n",
    "from torch.utils.data import random_split\n",
    "import torchvision\n",
    "from torchvision import transforms, datasets\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define transformations for train\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(p=.40),\n",
    "    transforms.RandomRotation(30),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define transformations for test\n",
    "test_transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define training dataloader\n",
    "def get_training_dataloader(train_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        train_transform: transfroms for train dataset\n",
    "        path: path to cifar100 training python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: train_data_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_train = train_transform\n",
    "    cifar10_training = torchvision.datasets.CIFAR10(root='.', train=True, download=True, transform=transform_train)\n",
    "    cifar10_training_loader = DataLoader(\n",
    "        cifar10_training, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_training_loader\n",
    "\n",
    "# define test dataloader\n",
    "def get_testing_dataloader(test_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        test_transform: transforms for test dataset\n",
    "        path: path to cifar100 test python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: cifar100_test_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_test = test_transform\n",
    "    cifar10_test = torchvision.datasets.CIFAR10(root='.', train=False, download=True, transform=transform_test)\n",
    "    cifar10_test_loader = DataLoader(\n",
    "        cifar10_test, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement mish activation function\n",
    "def f_mish(input):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "    '''\n",
    "    return input * torch.tanh(F.softplus(input))\n",
    "\n",
    "# implement class wrapper for mish activation function\n",
    "class mish(nn.Module):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = mish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_mish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement swish activation function\n",
    "def f_swish(input):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "    '''\n",
    "    return input * torch.sigmoid(input)\n",
    "\n",
    "# implement class wrapper for swish activation function\n",
    "class swish(nn.Module):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = swish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_swish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def channel_split(x, split):\n",
    "    \"\"\"split a tensor into two pieces along channel dimension\n",
    "    Args:\n",
    "        x: input tensor\n",
    "        split:(int) channel size for each pieces\n",
    "    \"\"\"\n",
    "    assert x.size(1) == split * 2\n",
    "    return torch.split(x, split, dim=1)\n",
    "    \n",
    "def channel_shuffle(x, groups):\n",
    "    \"\"\"channel shuffle operation\n",
    "    Args:\n",
    "        x: input tensor\n",
    "        groups: input branch number\n",
    "    \"\"\"\n",
    "\n",
    "    batch_size, channels, height, width = x.size()\n",
    "    channels_per_group = int(channels / groups)\n",
    "\n",
    "    x = x.view(batch_size, groups, channels_per_group, height, width)\n",
    "    x = x.transpose(1, 2).contiguous()\n",
    "    x = x.view(batch_size, -1, height, width)\n",
    "\n",
    "    return x\n",
    "\n",
    "class ShuffleUnit(nn.Module):\n",
    "\n",
    "    def __init__(self, in_channels, out_channels, stride, activation = 'relu'):\n",
    "        super().__init__()\n",
    "\n",
    "        self.stride = stride\n",
    "        self.in_channels = in_channels\n",
    "        self.out_channels = out_channels\n",
    "        self.activation = activation\n",
    "        \n",
    "        if (activation == 'relu'):\n",
    "            self.f_activation = nn.ReLU(inplace=True)\n",
    "            \n",
    "        if (activation == 'swish'):\n",
    "            self.f_activation = swish()\n",
    "            \n",
    "        if (activation == 'mish'):\n",
    "            self.f_activation = mish()\n",
    "\n",
    "        if stride != 1 or in_channels != out_channels:\n",
    "            self.residual = nn.Sequential(\n",
    "                nn.Conv2d(in_channels, in_channels, 1),\n",
    "                nn.BatchNorm2d(in_channels),\n",
    "                self.f_activation,\n",
    "                nn.Conv2d(in_channels, in_channels, 3, stride=stride, padding=1, groups=in_channels),\n",
    "                nn.BatchNorm2d(in_channels),\n",
    "                nn.Conv2d(in_channels, int(out_channels / 2), 1),\n",
    "                nn.BatchNorm2d(int(out_channels / 2)),\n",
    "                self.f_activation\n",
    "            )\n",
    "\n",
    "            self.shortcut = nn.Sequential(\n",
    "                nn.Conv2d(in_channels, in_channels, 3, stride=stride, padding=1, groups=in_channels),\n",
    "                nn.BatchNorm2d(in_channels),\n",
    "                nn.Conv2d(in_channels, int(out_channels / 2), 1),\n",
    "                nn.BatchNorm2d(int(out_channels / 2)),\n",
    "                self.f_activation\n",
    "            )\n",
    "        else:\n",
    "            self.shortcut = nn.Sequential()\n",
    "\n",
    "            in_channels = int(in_channels / 2)\n",
    "            self.residual = nn.Sequential(\n",
    "                nn.Conv2d(in_channels, in_channels, 1),\n",
    "                nn.BatchNorm2d(in_channels),\n",
    "                self.f_activation,\n",
    "                nn.Conv2d(in_channels, in_channels, 3, stride=stride, padding=1, groups=in_channels),\n",
    "                nn.BatchNorm2d(in_channels),\n",
    "                nn.Conv2d(in_channels, in_channels, 1),\n",
    "                nn.BatchNorm2d(in_channels),\n",
    "                self.f_activation \n",
    "            )\n",
    "\n",
    "    \n",
    "    def forward(self, x):\n",
    "\n",
    "        if self.stride == 1 and self.out_channels == self.in_channels:\n",
    "            shortcut, residual = channel_split(x, int(self.in_channels / 2))\n",
    "        else:\n",
    "            shortcut = x\n",
    "            residual = x\n",
    "        \n",
    "        shortcut = self.shortcut(shortcut)\n",
    "        residual = self.residual(residual)\n",
    "        x = torch.cat([shortcut, residual], dim=1)\n",
    "        x = channel_shuffle(x, 2)\n",
    "        \n",
    "        return x\n",
    "\n",
    "class ShuffleNetV2(nn.Module):\n",
    "\n",
    "    def __init__(self, ratio=1, class_num=10, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        if ratio == 0.5:\n",
    "            out_channels = [48, 96, 192, 1024]\n",
    "        elif ratio == 1:\n",
    "            out_channels = [116, 232, 464, 1024]\n",
    "        elif ratio == 1.5:\n",
    "            out_channels = [176, 352, 704, 1024]\n",
    "        elif ratio == 2:\n",
    "            out_channels = [244, 488, 976, 2048]\n",
    "        else:\n",
    "            ValueError('unsupported ratio number')\n",
    "            \n",
    "        if (activation == 'relu'):\n",
    "            self.f_activation = nn.ReLU(inplace=True)\n",
    "            \n",
    "        if (activation == 'swish'):\n",
    "            self.f_activation = swish()\n",
    "            \n",
    "        if (activation == 'mish'):\n",
    "            self.f_activation = mish()\n",
    "        \n",
    "        self.pre = nn.Sequential(\n",
    "            nn.Conv2d(3, 24, 3, padding=1),\n",
    "            nn.BatchNorm2d(24)\n",
    "        )\n",
    "\n",
    "        self.stage2 = self._make_stage(24, out_channels[0], 3, activation = activation)\n",
    "        self.stage3 = self._make_stage(out_channels[0], out_channels[1], 7, activation = activation)\n",
    "        self.stage4 = self._make_stage(out_channels[1], out_channels[2], 3, activation = activation)\n",
    "        self.conv5 = nn.Sequential(\n",
    "            nn.Conv2d(out_channels[2], out_channels[3], 1),\n",
    "            nn.BatchNorm2d(out_channels[3]),\n",
    "            self.f_activation\n",
    "        )\n",
    "\n",
    "        self.fc = nn.Linear(out_channels[3], class_num)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pre(x)\n",
    "        x = self.stage2(x)\n",
    "        x = self.stage3(x)\n",
    "        x = self.stage4(x)\n",
    "        x = self.conv5(x)\n",
    "        x = F.adaptive_avg_pool2d(x, 1)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.fc(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "    def _make_stage(self, in_channels, out_channels, repeat, activation = 'relu'):\n",
    "        layers = []\n",
    "        layers.append(ShuffleUnit(in_channels, out_channels, 2, activation = activation))\n",
    "\n",
    "        while repeat:\n",
    "            layers.append(ShuffleUnit(out_channels, out_channels, 1, activation = activation))\n",
    "            repeat -= 1\n",
    "        \n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "def shufflenetv2(activation = 'relu'):\n",
    "    return ShuffleNetV2(activation = activation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "170500096it [00:06, 27856084.73it/s]                               \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainloader = get_training_dataloader(train_transform)\n",
    "testloader = get_testing_dataloader(test_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epochs = 100\n",
    "batch_size = 128\n",
    "learning_rate = 0.001\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = shufflenetv2(activation = 'mish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set loss function\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# set optimizer, only train the classifier parameters, feature parameters are frozen\n",
    "optimizer = Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats = pd.DataFrame(columns = ['Epoch', 'Time per epoch', 'Avg time per step', 'Train loss', 'Train accuracy', 'Train top-3 accuracy','Test loss', 'Test accuracy', 'Test top-3 accuracy']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100.. Time per epoch: 36.6266.. Average time per step: 0.0937.. Train loss: 1.5088.. Train accuracy: 0.4411.. Top-3 train accuracy: 0.7787.. Test loss: 1.2205.. Test accuracy: 0.5605.. Top-3 test accuracy: 0.8646\n",
      "Epoch 2/100.. Time per epoch: 36.1809.. Average time per step: 0.0925.. Train loss: 1.1409.. Train accuracy: 0.5913.. Top-3 train accuracy: 0.8734.. Test loss: 0.9916.. Test accuracy: 0.6424.. Top-3 test accuracy: 0.9021\n",
      "Epoch 3/100.. Time per epoch: 35.7110.. Average time per step: 0.0913.. Train loss: 0.9624.. Train accuracy: 0.6593.. Top-3 train accuracy: 0.9038.. Test loss: 0.8600.. Test accuracy: 0.6985.. Top-3 test accuracy: 0.9254\n",
      "Epoch 4/100.. Time per epoch: 35.6949.. Average time per step: 0.0913.. Train loss: 0.8541.. Train accuracy: 0.6980.. Top-3 train accuracy: 0.9228.. Test loss: 0.7779.. Test accuracy: 0.7278.. Top-3 test accuracy: 0.9368\n",
      "Epoch 5/100.. Time per epoch: 35.7902.. Average time per step: 0.0915.. Train loss: 0.7797.. Train accuracy: 0.7262.. Top-3 train accuracy: 0.9317.. Test loss: 0.6869.. Test accuracy: 0.7615.. Top-3 test accuracy: 0.9461\n",
      "Epoch 6/100.. Time per epoch: 35.8502.. Average time per step: 0.0917.. Train loss: 0.7182.. Train accuracy: 0.7495.. Top-3 train accuracy: 0.9416.. Test loss: 0.6580.. Test accuracy: 0.7787.. Top-3 test accuracy: 0.9491\n",
      "Epoch 7/100.. Time per epoch: 35.6950.. Average time per step: 0.0913.. Train loss: 0.6783.. Train accuracy: 0.7620.. Top-3 train accuracy: 0.9469.. Test loss: 0.6252.. Test accuracy: 0.7831.. Top-3 test accuracy: 0.9518\n",
      "Epoch 8/100.. Time per epoch: 35.6046.. Average time per step: 0.0911.. Train loss: 0.6394.. Train accuracy: 0.7758.. Top-3 train accuracy: 0.9517.. Test loss: 0.5873.. Test accuracy: 0.7941.. Top-3 test accuracy: 0.9605\n",
      "Epoch 9/100.. Time per epoch: 35.4520.. Average time per step: 0.0907.. Train loss: 0.6066.. Train accuracy: 0.7881.. Top-3 train accuracy: 0.9551.. Test loss: 0.5594.. Test accuracy: 0.8113.. Top-3 test accuracy: 0.9600\n",
      "Epoch 10/100.. Time per epoch: 35.3527.. Average time per step: 0.0904.. Train loss: 0.5755.. Train accuracy: 0.7999.. Top-3 train accuracy: 0.9589.. Test loss: 0.5547.. Test accuracy: 0.8123.. Top-3 test accuracy: 0.9614\n",
      "Epoch 11/100.. Time per epoch: 35.6037.. Average time per step: 0.0911.. Train loss: 0.5504.. Train accuracy: 0.8068.. Top-3 train accuracy: 0.9616.. Test loss: 0.5530.. Test accuracy: 0.8069.. Top-3 test accuracy: 0.9607\n",
      "Epoch 12/100.. Time per epoch: 35.5222.. Average time per step: 0.0908.. Train loss: 0.5279.. Train accuracy: 0.8158.. Top-3 train accuracy: 0.9639.. Test loss: 0.5423.. Test accuracy: 0.8173.. Top-3 test accuracy: 0.9622\n",
      "Epoch 13/100.. Time per epoch: 35.8474.. Average time per step: 0.0917.. Train loss: 0.5009.. Train accuracy: 0.8265.. Top-3 train accuracy: 0.9675.. Test loss: 0.5370.. Test accuracy: 0.8188.. Top-3 test accuracy: 0.9668\n",
      "Epoch 14/100.. Time per epoch: 35.8687.. Average time per step: 0.0917.. Train loss: 0.4829.. Train accuracy: 0.8328.. Top-3 train accuracy: 0.9683.. Test loss: 0.5024.. Test accuracy: 0.8251.. Top-3 test accuracy: 0.9674\n",
      "Epoch 15/100.. Time per epoch: 35.7216.. Average time per step: 0.0914.. Train loss: 0.4662.. Train accuracy: 0.8368.. Top-3 train accuracy: 0.9703.. Test loss: 0.5111.. Test accuracy: 0.8267.. Top-3 test accuracy: 0.9672\n",
      "Epoch 16/100.. Time per epoch: 35.8386.. Average time per step: 0.0917.. Train loss: 0.4512.. Train accuracy: 0.8419.. Top-3 train accuracy: 0.9728.. Test loss: 0.4888.. Test accuracy: 0.8305.. Top-3 test accuracy: 0.9699\n",
      "Epoch 17/100.. Time per epoch: 35.7708.. Average time per step: 0.0915.. Train loss: 0.4336.. Train accuracy: 0.8484.. Top-3 train accuracy: 0.9742.. Test loss: 0.5044.. Test accuracy: 0.8354.. Top-3 test accuracy: 0.9679\n",
      "Epoch 18/100.. Time per epoch: 35.6420.. Average time per step: 0.0912.. Train loss: 0.4133.. Train accuracy: 0.8558.. Top-3 train accuracy: 0.9771.. Test loss: 0.4765.. Test accuracy: 0.8444.. Top-3 test accuracy: 0.9716\n",
      "Epoch 19/100.. Time per epoch: 35.9867.. Average time per step: 0.0920.. Train loss: 0.4044.. Train accuracy: 0.8568.. Top-3 train accuracy: 0.9781.. Test loss: 0.4972.. Test accuracy: 0.8379.. Top-3 test accuracy: 0.9684\n",
      "Epoch 20/100.. Time per epoch: 35.8758.. Average time per step: 0.0918.. Train loss: 0.3895.. Train accuracy: 0.8638.. Top-3 train accuracy: 0.9798.. Test loss: 0.4848.. Test accuracy: 0.8421.. Top-3 test accuracy: 0.9709\n",
      "Epoch 21/100.. Time per epoch: 35.8965.. Average time per step: 0.0918.. Train loss: 0.3753.. Train accuracy: 0.8675.. Top-3 train accuracy: 0.9804.. Test loss: 0.4616.. Test accuracy: 0.8495.. Top-3 test accuracy: 0.9724\n",
      "Epoch 22/100.. Time per epoch: 36.0436.. Average time per step: 0.0922.. Train loss: 0.3615.. Train accuracy: 0.8744.. Top-3 train accuracy: 0.9815.. Test loss: 0.4717.. Test accuracy: 0.8454.. Top-3 test accuracy: 0.9714\n",
      "Epoch 23/100.. Time per epoch: 36.0201.. Average time per step: 0.0921.. Train loss: 0.3537.. Train accuracy: 0.8755.. Top-3 train accuracy: 0.9823.. Test loss: 0.4519.. Test accuracy: 0.8499.. Top-3 test accuracy: 0.9737\n",
      "Epoch 24/100.. Time per epoch: 36.1649.. Average time per step: 0.0925.. Train loss: 0.3460.. Train accuracy: 0.8797.. Top-3 train accuracy: 0.9833.. Test loss: 0.4725.. Test accuracy: 0.8478.. Top-3 test accuracy: 0.9696\n",
      "Epoch 25/100.. Time per epoch: 35.7456.. Average time per step: 0.0914.. Train loss: 0.3344.. Train accuracy: 0.8812.. Top-3 train accuracy: 0.9837.. Test loss: 0.4620.. Test accuracy: 0.8526.. Top-3 test accuracy: 0.9721\n",
      "Epoch 26/100.. Time per epoch: 35.8120.. Average time per step: 0.0916.. Train loss: 0.3244.. Train accuracy: 0.8854.. Top-3 train accuracy: 0.9854.. Test loss: 0.4609.. Test accuracy: 0.8505.. Top-3 test accuracy: 0.9710\n",
      "Epoch 27/100.. Time per epoch: 35.9033.. Average time per step: 0.0918.. Train loss: 0.3121.. Train accuracy: 0.8897.. Top-3 train accuracy: 0.9862.. Test loss: 0.4422.. Test accuracy: 0.8543.. Top-3 test accuracy: 0.9750\n",
      "Epoch 28/100.. Time per epoch: 35.8772.. Average time per step: 0.0918.. Train loss: 0.3049.. Train accuracy: 0.8922.. Top-3 train accuracy: 0.9864.. Test loss: 0.4841.. Test accuracy: 0.8472.. Top-3 test accuracy: 0.9729\n",
      "Epoch 29/100.. Time per epoch: 35.8283.. Average time per step: 0.0916.. Train loss: 0.3008.. Train accuracy: 0.8947.. Top-3 train accuracy: 0.9854.. Test loss: 0.4483.. Test accuracy: 0.8577.. Top-3 test accuracy: 0.9734\n",
      "Epoch 30/100.. Time per epoch: 35.8602.. Average time per step: 0.0917.. Train loss: 0.2837.. Train accuracy: 0.9011.. Top-3 train accuracy: 0.9879.. Test loss: 0.4685.. Test accuracy: 0.8528.. Top-3 test accuracy: 0.9734\n",
      "Epoch 31/100.. Time per epoch: 35.8773.. Average time per step: 0.0918.. Train loss: 0.2792.. Train accuracy: 0.9021.. Top-3 train accuracy: 0.9883.. Test loss: 0.4546.. Test accuracy: 0.8589.. Top-3 test accuracy: 0.9742\n",
      "Epoch 32/100.. Time per epoch: 36.2246.. Average time per step: 0.0926.. Train loss: 0.2721.. Train accuracy: 0.9044.. Top-3 train accuracy: 0.9896.. Test loss: 0.4625.. Test accuracy: 0.8587.. Top-3 test accuracy: 0.9756\n",
      "Epoch 33/100.. Time per epoch: 35.9061.. Average time per step: 0.0918.. Train loss: 0.2644.. Train accuracy: 0.9067.. Top-3 train accuracy: 0.9891.. Test loss: 0.4495.. Test accuracy: 0.8626.. Top-3 test accuracy: 0.9730\n",
      "Epoch 34/100.. Time per epoch: 36.0741.. Average time per step: 0.0923.. Train loss: 0.2517.. Train accuracy: 0.9108.. Top-3 train accuracy: 0.9899.. Test loss: 0.4487.. Test accuracy: 0.8636.. Top-3 test accuracy: 0.9725\n",
      "Epoch 35/100.. Time per epoch: 36.0469.. Average time per step: 0.0922.. Train loss: 0.2500.. Train accuracy: 0.9127.. Top-3 train accuracy: 0.9898.. Test loss: 0.4686.. Test accuracy: 0.8581.. Top-3 test accuracy: 0.9745\n",
      "Epoch 36/100.. Time per epoch: 36.0907.. Average time per step: 0.0923.. Train loss: 0.2448.. Train accuracy: 0.9121.. Top-3 train accuracy: 0.9910.. Test loss: 0.4495.. Test accuracy: 0.8637.. Top-3 test accuracy: 0.9747\n",
      "Epoch 37/100.. Time per epoch: 35.9527.. Average time per step: 0.0920.. Train loss: 0.2358.. Train accuracy: 0.9165.. Top-3 train accuracy: 0.9919.. Test loss: 0.4585.. Test accuracy: 0.8613.. Top-3 test accuracy: 0.9746\n",
      "Epoch 38/100.. Time per epoch: 36.0931.. Average time per step: 0.0923.. Train loss: 0.2320.. Train accuracy: 0.9183.. Top-3 train accuracy: 0.9915.. Test loss: 0.4627.. Test accuracy: 0.8593.. Top-3 test accuracy: 0.9726\n",
      "Epoch 39/100.. Time per epoch: 35.8677.. Average time per step: 0.0917.. Train loss: 0.2283.. Train accuracy: 0.9197.. Top-3 train accuracy: 0.9917.. Test loss: 0.5031.. Test accuracy: 0.8518.. Top-3 test accuracy: 0.9708\n",
      "Epoch 40/100.. Time per epoch: 35.8936.. Average time per step: 0.0918.. Train loss: 0.2190.. Train accuracy: 0.9230.. Top-3 train accuracy: 0.9927.. Test loss: 0.4521.. Test accuracy: 0.8629.. Top-3 test accuracy: 0.9753\n",
      "Epoch 41/100.. Time per epoch: 35.7168.. Average time per step: 0.0913.. Train loss: 0.2139.. Train accuracy: 0.9233.. Top-3 train accuracy: 0.9932.. Test loss: 0.4680.. Test accuracy: 0.8650.. Top-3 test accuracy: 0.9750\n",
      "Epoch 42/100.. Time per epoch: 35.9220.. Average time per step: 0.0919.. Train loss: 0.2143.. Train accuracy: 0.9226.. Top-3 train accuracy: 0.9931.. Test loss: 0.4587.. Test accuracy: 0.8637.. Top-3 test accuracy: 0.9745\n",
      "Epoch 43/100.. Time per epoch: 35.8555.. Average time per step: 0.0917.. Train loss: 0.2012.. Train accuracy: 0.9286.. Top-3 train accuracy: 0.9937.. Test loss: 0.4732.. Test accuracy: 0.8677.. Top-3 test accuracy: 0.9764\n",
      "Epoch 44/100.. Time per epoch: 36.0773.. Average time per step: 0.0923.. Train loss: 0.2010.. Train accuracy: 0.9290.. Top-3 train accuracy: 0.9936.. Test loss: 0.4683.. Test accuracy: 0.8617.. Top-3 test accuracy: 0.9730\n",
      "Epoch 45/100.. Time per epoch: 35.9922.. Average time per step: 0.0921.. Train loss: 0.1973.. Train accuracy: 0.9302.. Top-3 train accuracy: 0.9944.. Test loss: 0.4807.. Test accuracy: 0.8606.. Top-3 test accuracy: 0.9758\n",
      "Epoch 46/100.. Time per epoch: 36.0854.. Average time per step: 0.0923.. Train loss: 0.1898.. Train accuracy: 0.9328.. Top-3 train accuracy: 0.9941.. Test loss: 0.4735.. Test accuracy: 0.8648.. Top-3 test accuracy: 0.9752\n",
      "Epoch 47/100.. Time per epoch: 36.1039.. Average time per step: 0.0923.. Train loss: 0.1814.. Train accuracy: 0.9360.. Top-3 train accuracy: 0.9952.. Test loss: 0.4809.. Test accuracy: 0.8608.. Top-3 test accuracy: 0.9754\n",
      "Epoch 48/100.. Time per epoch: 36.3931.. Average time per step: 0.0931.. Train loss: 0.1815.. Train accuracy: 0.9363.. Top-3 train accuracy: 0.9950.. Test loss: 0.5101.. Test accuracy: 0.8579.. Top-3 test accuracy: 0.9714\n",
      "Epoch 49/100.. Time per epoch: 35.9482.. Average time per step: 0.0919.. Train loss: 0.1839.. Train accuracy: 0.9359.. Top-3 train accuracy: 0.9947.. Test loss: 0.4696.. Test accuracy: 0.8613.. Top-3 test accuracy: 0.9755\n",
      "Epoch 50/100.. Time per epoch: 36.1055.. Average time per step: 0.0923.. Train loss: 0.1730.. Train accuracy: 0.9397.. Top-3 train accuracy: 0.9955.. Test loss: 0.4810.. Test accuracy: 0.8652.. Top-3 test accuracy: 0.9756\n",
      "Epoch 51/100.. Time per epoch: 36.0399.. Average time per step: 0.0922.. Train loss: 0.1681.. Train accuracy: 0.9410.. Top-3 train accuracy: 0.9955.. Test loss: 0.4977.. Test accuracy: 0.8641.. Top-3 test accuracy: 0.9751\n",
      "Epoch 52/100.. Time per epoch: 35.9669.. Average time per step: 0.0920.. Train loss: 0.1694.. Train accuracy: 0.9411.. Top-3 train accuracy: 0.9951.. Test loss: 0.4901.. Test accuracy: 0.8645.. Top-3 test accuracy: 0.9746\n",
      "Epoch 53/100.. Time per epoch: 35.9774.. Average time per step: 0.0920.. Train loss: 0.1663.. Train accuracy: 0.9404.. Top-3 train accuracy: 0.9958.. Test loss: 0.4848.. Test accuracy: 0.8689.. Top-3 test accuracy: 0.9754\n",
      "Epoch 54/100.. Time per epoch: 35.8773.. Average time per step: 0.0918.. Train loss: 0.1596.. Train accuracy: 0.9438.. Top-3 train accuracy: 0.9958.. Test loss: 0.5066.. Test accuracy: 0.8635.. Top-3 test accuracy: 0.9730\n",
      "Epoch 55/100.. Time per epoch: 35.7328.. Average time per step: 0.0914.. Train loss: 0.1582.. Train accuracy: 0.9440.. Top-3 train accuracy: 0.9959.. Test loss: 0.4868.. Test accuracy: 0.8669.. Top-3 test accuracy: 0.9749\n",
      "Epoch 56/100.. Time per epoch: 35.9619.. Average time per step: 0.0920.. Train loss: 0.1547.. Train accuracy: 0.9457.. Top-3 train accuracy: 0.9963.. Test loss: 0.4864.. Test accuracy: 0.8680.. Top-3 test accuracy: 0.9755\n",
      "Epoch 57/100.. Time per epoch: 35.6603.. Average time per step: 0.0912.. Train loss: 0.1526.. Train accuracy: 0.9466.. Top-3 train accuracy: 0.9967.. Test loss: 0.5046.. Test accuracy: 0.8615.. Top-3 test accuracy: 0.9753\n",
      "Epoch 58/100.. Time per epoch: 35.6362.. Average time per step: 0.0911.. Train loss: 0.1496.. Train accuracy: 0.9470.. Top-3 train accuracy: 0.9964.. Test loss: 0.4920.. Test accuracy: 0.8694.. Top-3 test accuracy: 0.9751\n",
      "Epoch 59/100.. Time per epoch: 36.1000.. Average time per step: 0.0923.. Train loss: 0.1462.. Train accuracy: 0.9486.. Top-3 train accuracy: 0.9967.. Test loss: 0.4944.. Test accuracy: 0.8645.. Top-3 test accuracy: 0.9779\n",
      "Epoch 60/100.. Time per epoch: 35.4951.. Average time per step: 0.0908.. Train loss: 0.1405.. Train accuracy: 0.9508.. Top-3 train accuracy: 0.9967.. Test loss: 0.5456.. Test accuracy: 0.8591.. Top-3 test accuracy: 0.9745\n",
      "Epoch 61/100.. Time per epoch: 35.6707.. Average time per step: 0.0912.. Train loss: 0.1413.. Train accuracy: 0.9499.. Top-3 train accuracy: 0.9970.. Test loss: 0.5017.. Test accuracy: 0.8656.. Top-3 test accuracy: 0.9770\n",
      "Epoch 62/100.. Time per epoch: 35.7495.. Average time per step: 0.0914.. Train loss: 0.1386.. Train accuracy: 0.9516.. Top-3 train accuracy: 0.9969.. Test loss: 0.5185.. Test accuracy: 0.8587.. Top-3 test accuracy: 0.9750\n",
      "Epoch 63/100.. Time per epoch: 35.9152.. Average time per step: 0.0919.. Train loss: 0.1330.. Train accuracy: 0.9533.. Top-3 train accuracy: 0.9975.. Test loss: 0.4970.. Test accuracy: 0.8679.. Top-3 test accuracy: 0.9780\n",
      "Epoch 64/100.. Time per epoch: 35.4510.. Average time per step: 0.0907.. Train loss: 0.1345.. Train accuracy: 0.9536.. Top-3 train accuracy: 0.9968.. Test loss: 0.5108.. Test accuracy: 0.8659.. Top-3 test accuracy: 0.9763\n",
      "Epoch 65/100.. Time per epoch: 35.4592.. Average time per step: 0.0907.. Train loss: 0.1347.. Train accuracy: 0.9525.. Top-3 train accuracy: 0.9974.. Test loss: 0.5169.. Test accuracy: 0.8706.. Top-3 test accuracy: 0.9745\n",
      "Epoch 66/100.. Time per epoch: 35.2784.. Average time per step: 0.0902.. Train loss: 0.1289.. Train accuracy: 0.9549.. Top-3 train accuracy: 0.9973.. Test loss: 0.5080.. Test accuracy: 0.8711.. Top-3 test accuracy: 0.9763\n",
      "Epoch 67/100.. Time per epoch: 35.3862.. Average time per step: 0.0905.. Train loss: 0.1231.. Train accuracy: 0.9567.. Top-3 train accuracy: 0.9976.. Test loss: 0.5125.. Test accuracy: 0.8676.. Top-3 test accuracy: 0.9767\n",
      "Epoch 68/100.. Time per epoch: 35.3352.. Average time per step: 0.0904.. Train loss: 0.1261.. Train accuracy: 0.9554.. Top-3 train accuracy: 0.9974.. Test loss: 0.5147.. Test accuracy: 0.8678.. Top-3 test accuracy: 0.9753\n",
      "Epoch 69/100.. Time per epoch: 35.4449.. Average time per step: 0.0907.. Train loss: 0.1229.. Train accuracy: 0.9568.. Top-3 train accuracy: 0.9976.. Test loss: 0.5300.. Test accuracy: 0.8658.. Top-3 test accuracy: 0.9738\n",
      "Epoch 70/100.. Time per epoch: 35.2943.. Average time per step: 0.0903.. Train loss: 0.1210.. Train accuracy: 0.9577.. Top-3 train accuracy: 0.9975.. Test loss: 0.5359.. Test accuracy: 0.8635.. Top-3 test accuracy: 0.9754\n",
      "Epoch 71/100.. Time per epoch: 35.4886.. Average time per step: 0.0908.. Train loss: 0.1169.. Train accuracy: 0.9583.. Top-3 train accuracy: 0.9980.. Test loss: 0.5132.. Test accuracy: 0.8686.. Top-3 test accuracy: 0.9762\n",
      "Epoch 72/100.. Time per epoch: 35.2671.. Average time per step: 0.0902.. Train loss: 0.1141.. Train accuracy: 0.9595.. Top-3 train accuracy: 0.9975.. Test loss: 0.5517.. Test accuracy: 0.8631.. Top-3 test accuracy: 0.9753\n",
      "Epoch 73/100.. Time per epoch: 35.1392.. Average time per step: 0.0899.. Train loss: 0.1156.. Train accuracy: 0.9581.. Top-3 train accuracy: 0.9978.. Test loss: 0.5210.. Test accuracy: 0.8655.. Top-3 test accuracy: 0.9763\n",
      "Epoch 74/100.. Time per epoch: 34.9669.. Average time per step: 0.0894.. Train loss: 0.1185.. Train accuracy: 0.9582.. Top-3 train accuracy: 0.9981.. Test loss: 0.5406.. Test accuracy: 0.8640.. Top-3 test accuracy: 0.9755\n",
      "Epoch 75/100.. Time per epoch: 35.1275.. Average time per step: 0.0898.. Train loss: 0.1074.. Train accuracy: 0.9624.. Top-3 train accuracy: 0.9982.. Test loss: 0.5511.. Test accuracy: 0.8667.. Top-3 test accuracy: 0.9763\n",
      "Epoch 76/100.. Time per epoch: 35.0270.. Average time per step: 0.0896.. Train loss: 0.1084.. Train accuracy: 0.9616.. Top-3 train accuracy: 0.9981.. Test loss: 0.5368.. Test accuracy: 0.8658.. Top-3 test accuracy: 0.9765\n",
      "Epoch 77/100.. Time per epoch: 35.2011.. Average time per step: 0.0900.. Train loss: 0.1034.. Train accuracy: 0.9630.. Top-3 train accuracy: 0.9982.. Test loss: 0.5371.. Test accuracy: 0.8668.. Top-3 test accuracy: 0.9777\n",
      "Epoch 78/100.. Time per epoch: 35.0453.. Average time per step: 0.0896.. Train loss: 0.1055.. Train accuracy: 0.9631.. Top-3 train accuracy: 0.9981.. Test loss: 0.5246.. Test accuracy: 0.8725.. Top-3 test accuracy: 0.9782\n",
      "Epoch 79/100.. Time per epoch: 35.2756.. Average time per step: 0.0902.. Train loss: 0.1041.. Train accuracy: 0.9633.. Top-3 train accuracy: 0.9981.. Test loss: 0.5230.. Test accuracy: 0.8711.. Top-3 test accuracy: 0.9765\n",
      "Epoch 80/100.. Time per epoch: 35.2586.. Average time per step: 0.0902.. Train loss: 0.1039.. Train accuracy: 0.9633.. Top-3 train accuracy: 0.9981.. Test loss: 0.5486.. Test accuracy: 0.8675.. Top-3 test accuracy: 0.9752\n",
      "Epoch 81/100.. Time per epoch: 35.2280.. Average time per step: 0.0901.. Train loss: 0.1055.. Train accuracy: 0.9628.. Top-3 train accuracy: 0.9983.. Test loss: 0.5298.. Test accuracy: 0.8695.. Top-3 test accuracy: 0.9764\n",
      "Epoch 82/100.. Time per epoch: 35.1085.. Average time per step: 0.0898.. Train loss: 0.0960.. Train accuracy: 0.9653.. Top-3 train accuracy: 0.9986.. Test loss: 0.5379.. Test accuracy: 0.8680.. Top-3 test accuracy: 0.9772\n",
      "Epoch 83/100.. Time per epoch: 35.0015.. Average time per step: 0.0895.. Train loss: 0.0981.. Train accuracy: 0.9653.. Top-3 train accuracy: 0.9982.. Test loss: 0.5604.. Test accuracy: 0.8672.. Top-3 test accuracy: 0.9756\n",
      "Epoch 84/100.. Time per epoch: 35.2430.. Average time per step: 0.0901.. Train loss: 0.0994.. Train accuracy: 0.9650.. Top-3 train accuracy: 0.9984.. Test loss: 0.5727.. Test accuracy: 0.8661.. Top-3 test accuracy: 0.9753\n",
      "Epoch 85/100.. Time per epoch: 35.1227.. Average time per step: 0.0898.. Train loss: 0.0999.. Train accuracy: 0.9649.. Top-3 train accuracy: 0.9983.. Test loss: 0.5517.. Test accuracy: 0.8699.. Top-3 test accuracy: 0.9760\n",
      "Epoch 86/100.. Time per epoch: 34.9969.. Average time per step: 0.0895.. Train loss: 0.0939.. Train accuracy: 0.9667.. Top-3 train accuracy: 0.9988.. Test loss: 0.5575.. Test accuracy: 0.8700.. Top-3 test accuracy: 0.9741\n",
      "Epoch 87/100.. Time per epoch: 35.1178.. Average time per step: 0.0898.. Train loss: 0.0955.. Train accuracy: 0.9658.. Top-3 train accuracy: 0.9986.. Test loss: 0.5705.. Test accuracy: 0.8665.. Top-3 test accuracy: 0.9752\n",
      "Epoch 88/100.. Time per epoch: 35.2415.. Average time per step: 0.0901.. Train loss: 0.0921.. Train accuracy: 0.9676.. Top-3 train accuracy: 0.9986.. Test loss: 0.5427.. Test accuracy: 0.8731.. Top-3 test accuracy: 0.9766\n",
      "Epoch 89/100.. Time per epoch: 35.0612.. Average time per step: 0.0897.. Train loss: 0.0930.. Train accuracy: 0.9671.. Top-3 train accuracy: 0.9986.. Test loss: 0.5695.. Test accuracy: 0.8661.. Top-3 test accuracy: 0.9769\n",
      "Epoch 90/100.. Time per epoch: 34.9364.. Average time per step: 0.0894.. Train loss: 0.0860.. Train accuracy: 0.9697.. Top-3 train accuracy: 0.9988.. Test loss: 0.5546.. Test accuracy: 0.8729.. Top-3 test accuracy: 0.9775\n",
      "Epoch 91/100.. Time per epoch: 34.8041.. Average time per step: 0.0890.. Train loss: 0.0889.. Train accuracy: 0.9690.. Top-3 train accuracy: 0.9986.. Test loss: 0.5745.. Test accuracy: 0.8675.. Top-3 test accuracy: 0.9743\n",
      "Epoch 92/100.. Time per epoch: 34.9494.. Average time per step: 0.0894.. Train loss: 0.0895.. Train accuracy: 0.9681.. Top-3 train accuracy: 0.9989.. Test loss: 0.5760.. Test accuracy: 0.8694.. Top-3 test accuracy: 0.9765\n",
      "Epoch 93/100.. Time per epoch: 34.9091.. Average time per step: 0.0893.. Train loss: 0.0867.. Train accuracy: 0.9692.. Top-3 train accuracy: 0.9987.. Test loss: 0.5542.. Test accuracy: 0.8676.. Top-3 test accuracy: 0.9754\n",
      "Epoch 94/100.. Time per epoch: 34.9098.. Average time per step: 0.0893.. Train loss: 0.0833.. Train accuracy: 0.9710.. Top-3 train accuracy: 0.9989.. Test loss: 0.5784.. Test accuracy: 0.8677.. Top-3 test accuracy: 0.9768\n",
      "Epoch 95/100.. Time per epoch: 35.0866.. Average time per step: 0.0897.. Train loss: 0.0865.. Train accuracy: 0.9694.. Top-3 train accuracy: 0.9989.. Test loss: 0.5608.. Test accuracy: 0.8724.. Top-3 test accuracy: 0.9745\n",
      "Epoch 96/100.. Time per epoch: 35.0588.. Average time per step: 0.0897.. Train loss: 0.0780.. Train accuracy: 0.9724.. Top-3 train accuracy: 0.9990.. Test loss: 0.5567.. Test accuracy: 0.8722.. Top-3 test accuracy: 0.9780\n",
      "Epoch 97/100.. Time per epoch: 35.0220.. Average time per step: 0.0896.. Train loss: 0.0839.. Train accuracy: 0.9707.. Top-3 train accuracy: 0.9990.. Test loss: 0.5827.. Test accuracy: 0.8707.. Top-3 test accuracy: 0.9754\n",
      "Epoch 98/100.. Time per epoch: 34.8398.. Average time per step: 0.0891.. Train loss: 0.0823.. Train accuracy: 0.9708.. Top-3 train accuracy: 0.9988.. Test loss: 0.5685.. Test accuracy: 0.8700.. Top-3 test accuracy: 0.9739\n",
      "Epoch 99/100.. Time per epoch: 35.0287.. Average time per step: 0.0896.. Train loss: 0.0825.. Train accuracy: 0.9716.. Top-3 train accuracy: 0.9987.. Test loss: 0.5655.. Test accuracy: 0.8717.. Top-3 test accuracy: 0.9770\n",
      "Epoch 100/100.. Time per epoch: 35.2261.. Average time per step: 0.0901.. Train loss: 0.0786.. Train accuracy: 0.9719.. Top-3 train accuracy: 0.9992.. Test loss: 0.5843.. Test accuracy: 0.8663.. Top-3 test accuracy: 0.9754\n"
     ]
    }
   ],
   "source": [
    "#train the model\n",
    "model.to(device)\n",
    "\n",
    "steps = 0\n",
    "running_loss = 0\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    since = time.time()\n",
    "    \n",
    "    train_accuracy = 0\n",
    "    top3_train_accuracy = 0 \n",
    "    for inputs, labels in trainloader:\n",
    "        steps += 1\n",
    "        # Move input and label tensors to the default device\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        logps = model.forward(inputs)\n",
    "        loss = criterion(logps, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "        # calculate train top-1 accuracy\n",
    "        ps = torch.exp(logps)\n",
    "        top_p, top_class = ps.topk(1, dim=1)\n",
    "        equals = top_class == labels.view(*top_class.shape)\n",
    "        train_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "        \n",
    "        # Calculate train top-3 accuracy\n",
    "        np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "        target_numpy = labels.cpu().numpy()\n",
    "        top3_train_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "        \n",
    "    time_elapsed = time.time() - since\n",
    "    \n",
    "    test_loss = 0\n",
    "    test_accuracy = 0\n",
    "    top3_test_accuracy = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in testloader:\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            logps = model.forward(inputs)\n",
    "            batch_loss = criterion(logps, labels)\n",
    "\n",
    "            test_loss += batch_loss.item()\n",
    "\n",
    "            # Calculate test top-1 accuracy\n",
    "            ps = torch.exp(logps)\n",
    "            top_p, top_class = ps.topk(1, dim=1)\n",
    "            equals = top_class == labels.view(*top_class.shape)\n",
    "            test_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "            \n",
    "            # Calculate test top-3 accuracy\n",
    "            np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "            target_numpy = labels.cpu().numpy()\n",
    "            top3_test_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "\n",
    "    print(f\"Epoch {epoch+1}/{epochs}.. \"\n",
    "          f\"Time per epoch: {time_elapsed:.4f}.. \"\n",
    "          f\"Average time per step: {time_elapsed/len(trainloader):.4f}.. \"\n",
    "          f\"Train loss: {running_loss/len(trainloader):.4f}.. \"\n",
    "          f\"Train accuracy: {train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Top-3 train accuracy: {top3_train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Test loss: {test_loss/len(testloader):.4f}.. \"\n",
    "          f\"Test accuracy: {test_accuracy/len(testloader):.4f}.. \"\n",
    "          f\"Top-3 test accuracy: {top3_test_accuracy/len(testloader):.4f}\")\n",
    "\n",
    "    train_stats = train_stats.append({'Epoch': epoch, 'Time per epoch':time_elapsed, 'Avg time per step': time_elapsed/len(trainloader), 'Train loss' : running_loss/len(trainloader), 'Train accuracy': train_accuracy/len(trainloader), 'Train top-3 accuracy':top3_train_accuracy/len(trainloader),'Test loss' : test_loss/len(testloader), 'Test accuracy': test_accuracy/len(testloader), 'Test top-3 accuracy':top3_test_accuracy/len(testloader)}, ignore_index=True)\n",
    "\n",
    "    running_loss = 0\n",
    "    model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats.to_csv('train_log_ShuffleNetv2_Mish.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
